{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TF-IDF算法关键词提取的结果：\n",
      "会徽 冰雪 书法 形态 字体 图形 运动员 中国 时代 梦想 "
     ]
    }
   ],
   "source": [
    "import math   #导入所需要的库与模块\n",
    "import jieba\n",
    "import jieba.posseg as psg\n",
    "#加载语料库并对语料库进行预处理\n",
    "def load_data(pos = False, corpus_path='./corpus.txt'):\n",
    "    doc_list = []\n",
    "    for line in open(corpus_path,'r',encoding = 'utf-8'):\n",
    "        content = line.strip()    #去除行首和行尾的空白字符\n",
    "        cut_list = cutWord(content, pos)     #对数据进行分词\n",
    "        filter_list = removeWord(cut_list, pos)    #删除停用词\n",
    "        doc_list.append(filter_list)\n",
    "    return doc_list\n",
    "#对文本进行分词\n",
    "def cutWord(sentence, pos = False):\n",
    "    if not pos:    #使用jieba库对文本进行分词\n",
    "        cut_list = jieba.cut(sentence)\n",
    "    else:\n",
    "        cut_list = psg.cut(sentence)\n",
    "    return cut_list\n",
    "#从分词列表中删除停用词\n",
    "def removeWord(seg_list, pos = False):\n",
    "    stop_word_path = './stops_list.txt'    #加载停用词表\n",
    "    stopword_list = [sw.replace('\\n', '') for sw in \n",
    "                     open(stop_word_path,'r',encoding='utf-8').readlines()]\n",
    "    filter_list=[]    #初始化一个空列表\n",
    "    for seg in seg_list:      #根据pos参数选择是否词性过滤\n",
    "        if not pos:\n",
    "            word = seg\n",
    "            flag = 'n'\n",
    "        else:\n",
    "            word = seg.word\n",
    "            flag = seg.flag\n",
    "        if not flag.startswith('n'):\n",
    "            continue\n",
    "        #删除停用词表中的词，以及长度小于2的词\n",
    "        if not word in stopword_list and len(word) > 1:\n",
    "            filter_list.append(word)\n",
    "    return filter_list\n",
    "#定义计算TF值的函数\n",
    "def get_tf(word_list):\n",
    "    tf_dic = {}\n",
    "    for word in word_list:\n",
    "        tf_dic[word] = tf_dic.get(word, 0.0) + 1.0\n",
    "    tt_count = len(word_list)     #计算文档的总词数\n",
    "    for k,v in tf_dic.items():\n",
    "        tf_dic[k] = float(v) / tt_count    #计算TF值\n",
    "    return tf_dic\n",
    "#定义计算IDF值的函数\n",
    "def get_idf(doc_list):\n",
    "    idf_dic= {}\n",
    "    tt_count = len(doc_list)    #计算总文档数\n",
    "    for doc in doc_list:       #遍历每篇文档\n",
    "        for word in set(doc):\n",
    "            idf_dic[word] = idf_dic.get(word, 0.0) + 1.0\n",
    "    for k,v in idf_dic.items():\n",
    "        idf_dic[k] = math.log(tt_count / (1.0 + v))\n",
    "    default_idf = math.log(tt_count / (1.0))  #计算IDF值\n",
    "    return idf_dic, default_idf\n",
    "#定义计算TF-IDF值的函数\n",
    "def get_tfidf(idf_dic, default_idf, word_list, keyword_num):\n",
    "    tfidf_dic = {}\n",
    "    for word in word_list:\n",
    "        idf = idf_dic.get(word,default_idf)  #计算每个词的IDF值\n",
    "        tf_dic = get_tf(word_list)\n",
    "        tf = tf_dic.get(word, 0)    #计算每个词的TF值\n",
    "        tfidf = tf * idf            #计算每个词的TF-IDF值\n",
    "        tfidf_dic[word] = tfidf\n",
    "    #根据TF-IDF值进行降序排列\n",
    "    for k,v in sorted(tfidf_dic.items(), key = lambda x:x[1],\n",
    "                     reverse = True)[:keyword_num]:\n",
    "        print(k + \" \", end = '')\n",
    "#计算TF-IDF值\n",
    "def tfidf_extract(word_list, pos = False, keyword_num = 10):\n",
    "    doc_list = load_data(pos)  #加载语料库并对语料库进行预处理\n",
    "    idf_dic, default_idf = get_idf(doc_list)  #计算IDF值\n",
    "    tfidf_model = get_tfidf(idf_dic, default_idf, word_list, keyword_num)\n",
    "if __name__ == '__main__':\n",
    "    text ='冬奥会会徽以汉字“冬”为灵感来源，运用中国书法的艺术形态， 将厚重的东方文化底蕴与国际化的现代风格融为一体，呈现出新时代的中国新形象、新梦想，传递出新时代中国为办好北京冬奥会，圆冬奥之梦，实现“三亿人参与冰雪运动”目标，圆体育强国之梦，推动世界冰雪运动发展，为国际奥林匹克运动做出新贡献的不懈努力和美好追求。会徽图形上半部分展现滑冰运动员的造型，下半部分表现滑雪运动员的英姿。中间舞动的线条流畅且充满韵律，代表举办地起伏的山峦、赛场、冰雪滑道和节日飘舞的丝带，为会徽增添了节日喜庆的视觉感受，也象征着北京冬奥会将在中国春节期间举行。会徽以蓝色为主色调，寓意梦想与未来，以及冰雪的明亮纯洁。红黄两色源自中国国旗，代表运动的激情、青春与活力。在“BEIJING 2022”字体的形态上汲取了中国书法与剪纸的特点，增强了字体的文化内涵和表现力，也体现了与会徽图形的整体感和统一性。'\n",
    "    cut_list = cutWord(text, pos = True)   #分词并标注词性\n",
    "    filter_list = removeWord(cut_list, pos = True)  #删除停用词\n",
    "    print('TF-IDF算法关键词提取的结果：')\n",
    "    tfidf_extract(filter_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
